package com.example.redis.spider;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import us.codecraft.webmagic.*;
import us.codecraft.webmagic.pipeline.Pipeline;
import us.codecraft.webmagic.processor.PageProcessor;

import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;


public class CSDNPageProcessor implements PageProcessor {
    private Logger logger= LoggerFactory.getLogger(CSDNPageProcessor.class);
    private Site site = Site
            .me()
            .setDomain("csdn.net")
            .setSleepTime(300)
            .setUserAgent(
                    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.65 Safari/537.31");

    public static void main(String[] args) {
        Spider.create(new CSDNPageProcessor()).addUrl("https://blog.csdn.net/nav/java")
                .thread(5)
                .addPipeline(new Pipeline() {
                    @Override
                    public void process(ResultItems resultItems, Task task) {
                        String path = "d:/webmagic/";
                        try {
                            PrintWriter printWriter = new PrintWriter(new FileWriter(path+resultItems.get("title") + ".html"));
                            printWriter.write(resultItems.get("content").toString());
                            printWriter.close();
                        } catch (IOException e) {

                        }
                    }
                })
                .run();
    }

    @Override
    public void process(Page page) {
//        List<String> all = page.getHtml().links().regex("https://blog.csdn.net/\\w+/article/details/\\w+").all();
//        List<String> all = page.getHtml().xpath("//div[@class='title']//h2//a/@href").all();
        page.addTargetRequests(UrlGetter.getUrl());
        page.putField("title", page.getHtml().xpath("//h1[@class='title-article']/text()"));
        page.putField("author", page.getHtml().xpath("//div[@class='bar-content']/a/text()"));
        page.putField("content", page.getHtml().xpath("//div[@class='article_content clearfix']"));
    }

    @Override
    public Site getSite() {
        return site;
    }
}
